notebook.community

Edit and run



In [1]:

    
import itertools



In [9]:

    
import numpy as np

Figure out how to divide an iterable into evenly-sized chunks.



In [3]:

    
ids = ('C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9')

Suppose the iterable is a tuple of all possible pairs from ids:



In [4]:

    
pairs = tuple(itertools.combinations(ids, 2))



In [5]:

    
len(pairs)









    Out[5]:





36

Also suppose that there are 16 processors, each operating on a single chunk.



In [6]:

    
numProc = 16



In [2]:

    
def get_chunk_lengths(vec, numProc):
    """Get the length of each chunk"""
    numDivFloor = len(vec)//numProc
    lengths = [numDivFloor]*numProc
    remainder = len(vec) - numDivFloor*numProc
    if remainder > len(vec):
        print('Error: remainder greater than vector length')
        return
    else:
        for i in range(remainder):
            lengths[i] += 1
        return lengths



In [7]:

    
splitlengths = get_chunk_lengths(pairs, numProc)



In [8]:

    
splitlengths









    Out[8]:





[3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]



In [16]:

    
divpos = np.cumsum([0] + splitlengths)



In [17]:

    
divpos









    Out[17]:





array([ 0,  3,  6,  9, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36])



In [18]:

    
len(divpos)









    Out[18]:





17



In [22]:

    
def chunks(divpos, vec):
    for i in range(len(divpos)-1):
        yield vec[divpos[i]:divpos[i+1]]



In [23]:

    
tuple(chunks(divpos, pairs))









    Out[23]:





((('C1', 'C2'), ('C1', 'C3'), ('C1', 'C4')),
 (('C1', 'C5'), ('C1', 'C6'), ('C1', 'C7')),
 (('C1', 'C8'), ('C1', 'C9'), ('C2', 'C3')),
 (('C2', 'C4'), ('C2', 'C5'), ('C2', 'C6')),
 (('C2', 'C7'), ('C2', 'C8')),
 (('C2', 'C9'), ('C3', 'C4')),
 (('C3', 'C5'), ('C3', 'C6')),
 (('C3', 'C7'), ('C3', 'C8')),
 (('C3', 'C9'), ('C4', 'C5')),
 (('C4', 'C6'), ('C4', 'C7')),
 (('C4', 'C8'), ('C4', 'C9')),
 (('C5', 'C6'), ('C5', 'C7')),
 (('C5', 'C8'), ('C5', 'C9')),
 (('C6', 'C7'), ('C6', 'C8')),
 (('C6', 'C9'), ('C7', 'C8')),
 (('C7', 'C9'), ('C8', 'C9')))

Try it on an iterable that is the range of integers from 10 to 74, with number of processors equal to 7:



In [28]:

    
numProc = 7
v = tuple(range(10,75))
splitLengths = get_chunk_lengths(v, numProc)
divpos = np.cumsum([0] + splitLengths)
tuple(chunks(divpos, v))









    Out[28]:





((10, 11, 12, 13, 14, 15, 16, 17, 18, 19),
 (20, 21, 22, 23, 24, 25, 26, 27, 28, 29),
 (30, 31, 32, 33, 34, 35, 36, 37, 38),
 (39, 40, 41, 42, 43, 44, 45, 46, 47),
 (48, 49, 50, 51, 52, 53, 54, 55, 56),
 (57, 58, 59, 60, 61, 62, 63, 64, 65),
 (66, 67, 68, 69, 70, 71, 72, 73, 74))



In [ ]: